import datetime
import os
import pandas as pd
import Core.WindFunctions as Wind
from SystematicFactors.General import *
# from SystematicFactors.SysFactors_Basic import *
from SystematicFactors.SysFactors_Currency import *
from SystematicFactors.SysFactors_Credit_Expansion import *
from SystematicFactors.SysFactors_Capital_Flow import *
from SystematicFactors.SysFactors_Equity_Market_Technical import *
from SystematicFactors.SysFactors_Equity_Market_Fundamental import *
from SystematicFactors.SysFactors_Bond_Market import *
from SystematicFactors.SysFactors_Economics import *
from SystematicFactors.SysFactors_Advanced import *
from SystematicFactors.SysFactors_ExternalFiles import *
from SystematicFactors.SysFactors_ZYYX import *
from SystematicFactors.DoomsdayClock import *
import SystematicFactors.ReturnStructure
import SystematicFactors.Basis
import SystematicFactors.Basis_v2
import SystematicFactors.DDM_Cycle
import SystematicFactors.Kitchin_Cycle
import SystematicFactors.Hurst
import SystematicFactors.iVixFactor
from DataProcess.DownloadMacroData import Download_Risk_Free_Rate


def Generate_Systematic_Factor_Dataframe(datetime1, datetime2, strategy_category):
    #
    df = Gadget.Generate_Calender_Days_DataFrame(datetime1.date(), datetime2.date(), date_field_name="date")
    df["date_t"] = pd.to_datetime(df["date"])
    df.set_index("date_t", inplace=True)
    #
    if strategy_category == "CHNBOND":
        df = df.resample("M").last()
    elif strategy_category == "HS300":
        df = df.resample("W").last()
    print(df)
    print("Count", len(df))

    documents_factor = database.Find("Factor", "a_sys_factor_map", {"strategy_category": strategy_category}, sort=[("factor_id", 1)])
    for document_factor in documents_factor:
        factor_name = document_factor["factor_name"]
        former_name = document_factor["former_name"]
        #
        date_field = "report_date"
        #
        if factor_name in ["Investor_Confi_Global_Monthly_lnDif"]:
            continue
        #
        if factor_name in ["Investor_Confi_Monthly_lnDif", "Investor_Confi_Valuation_Monthly_lnDif"]:
            date_field = "date"

        #
        documents = database.Find("Factor", "a_sys_factor", filter={"name": factor_name}, sort=[("Date",1)])
        if len(documents) == 0:
            print(factor_name, "No Data")
            continue
        #
        print(documents[0]["date"], documents[-1]["date"], "#", len(documents), factor_name)

        # Generate Dataframe
        data = []
        for value in documents:
            data.append([value[date_field], value["value"]])
        df_factor = pd.DataFrame(data, columns=["date", former_name])

        #
        df_factor["date"] = pd.to_datetime(df_factor["date"])
        df = pd.merge(df, df_factor, how="left", left_on="date", right_on="date")
        print("Count", len(df))
        a = 0

    df.to_csv("d://" + strategy_category + ".csv")
    return df


# 批量计算系统层因子
def Automatic_Calculate_Systematic_Factors(database, realtime, datetime2, cold_start=False):
    #
    datetime1 = datetime2 + datetime.timedelta(days=-210)
    datetime1 = datetime.datetime(datetime1.year, datetime1.month, datetime1.day)
    if cold_start:
        datetime1 = datetime.datetime(2000,1,1)
    #
    datetime2 = datetime2 + datetime.timedelta(days=1)
    next_month = datetime2 + datetime.timedelta(days=30)

    # 确保从一个周一开始计算，避免周中开始的数据，周度聚合时会出现数据不完整的现象
    datetime1 = Gadget.Find_Recent_Weekday(datetime1, 1)

    # ---货币现象---
    # 央行与货币政策
    Calc_OMO_Central_Bank_Operation(database, datetime1, datetime2)  # 央行货币政策
    Calc_OMO_Slf_Mlf_Psl(database, datetime1, datetime2)
    Calc_Excess_Reserve_Ratio(database, datetime1, datetime2)
    # Calc_Deposit_Reserve_Ratio(database, datetime1, datetime2)

    # ---货币市场：人民币与外汇---
    Calc_FX(database, datetime1, datetime2)
    Calc_Fx_Reserve(database, datetime1, datetime2)
    Calc_FX_Return(database, datetime1, datetime2)
    Calc_CHN_US_IR_Spread(database, datetime1, datetime2)
    Calc_CHN_US_IR_Spread_Month_Avg(database, datetime1, datetime2)
    Calc_Hotmoney(database, datetime1, datetime2)

    # ---信用现象---
    Calc_Newloan(database, datetime1, datetime2)
    Calc_Newloan_Residuals(database, datetime1, datetime2)
    Calc_Newloan_Short_Term(database, datetime1, datetime2)
    Calc_Newloan_Long_Term(database, datetime1, datetime2)
    Calc_Govt_Leverage(database, datetime1, datetime2)
    Calc_Money_Multiplier(database, datetime1, datetime2)
    Calc_M0(database, datetime1, datetime2)
    Calc_M2_M1(database, datetime1, datetime2)
    Calc_Investable_Capital(database, datetime1, datetime2)
    Calc_Liquidity_Surplus(database, datetime1, datetime2)
    Calc_Bank_Surplus(database, datetime1, datetime2)
    Calc_Deposit_Loan_Ratio(database, datetime1, datetime2)
    Calc_Social_Financing(database, datetime1, datetime2)

    # ---经济现象---
    # 经济预期
    Calc_PMI(database, datetime1, datetime2) # 经济：PMI
    download_bci_indicator(database, datetime1, datetime2)
    # 经济：工业企业
    Calc_Industrial_Liability_Ratio(database, datetime1, datetime2)  #
    Calc_Industrial_Profmargin(database, datetime1, datetime2)
    Calc_Industrial_ATO(database, datetime1, datetime2) # 需要改造
    Calc_Industrial_Added(database, datetime1, datetime2)
    Calc_RealEstate(database, datetime1, datetime2) # 经济：地产
    Calc_Fixed_Asset_Invested(database, datetime1, datetime2)   # 经济：投资
    Calc_Consumption(database, datetime1, datetime2)    # 经济：消费
    Calc_Net_Export_Ratio(database, datetime1, datetime2)   # 经济：出口

    # ---价格现象---
    Calc_Price_Level(database, datetime1, datetime2)

    # ---债券市场---
    # 资金市场
    Calc_Money_Market(database, datetime1, datetime2)  # 资金市场
    Calc_MM_Repo(database, datetime1, datetime2)  # 数值对不上 日数据取月末
    Calc_MM_Repo_SD(database, datetime1, datetime2)  # rolling 把时间补满
    Calc_Shibor_Avg(database, datetime1, datetime2)
    Download_Risk_Free_Rate(database, datetime1, datetime2)

    # 债券市场
    Calc_Bond_Market_Rate(database, datetime1, datetime2)  # 债券市场
    Calc_Term_Spread(database, datetime1, datetime2)
    Calc_Credit_Spread(database, datetime1, datetime2)
    Calc_Credit_Margin_Guokai(database, datetime1, datetime2)
    Calc_Credit_Margin_AAA(database, datetime1, datetime2)
    Calc_Ted_Spread(database, datetime1, datetime2)
    download_swap_rate(database, datetime1, datetime2)

    # 影子银行
    Calc_Licai_Market(database, datetime1, datetime2)  # 理财市场
    Calc_Trust(database, datetime1, datetime2)

    # ---股票市场：基本面---
    Calc_PE_Level(database, datetime1, datetime2)
    Calc_Dividend_To_TBond_Yield(database, datetime1, datetime2)
    Calc_EPS_Chg_To_Amt_Chg(database, datetime1, datetime2)
    download_equity_index_valuation_factor(database, datetime1, datetime2)

    # --- 股票市场：资金面供给 ---
    Calc_Capital_In(database, datetime1, datetime2)  # 沪深300净买入
    download_money_flow(database, datetime1, datetime2)

    # 新增投资者
    Calc_New_Investor_old(database, datetime1, datetime2)
    Calc_New_Investor(database, datetime1, datetime2)  # 新增投资者 @ 20238月终止
    Calc_New_MarginTrade_Investor(database, datetime1, datetime2)  # 新增融资融券客户
    #
    Calc_Margin_Trade(database, datetime1, datetime2)  # 融资融券相关

    Calc_New_MF_Equity_Shares(database, datetime1, datetime2)
    Calc_Fund_Aum(database, datetime1, datetime2)
    # Calc_Big_HedgeFund(database, datetime1, datetime2) 2020已经停止
    #
    Calc_Stock_Connect(database, datetime1, datetime2)  # 港股通
    Calc_Deposit_To_Market(database, datetime1, datetime2)  # 储蓄市值比

    # ---股票市场：资金需求---
    # 股权融资
    Calc_IPO(database, datetime1, next_month)  # IPO融资
    Calc_SEO(database, datetime1, next_month)  # 增发
    Calc_Equity_Issue(database, datetime1, datetime2)  # 股权融资规模

    # 债券融资
    Calc_Bond_Net_Issue_Sovereign(database, datetime1, datetime2)  # 主权债净发行
    Calc_Bond_Net_Issue_Credit(database, datetime1, datetime2)  # 信用债净发行

    # 二级市场增减持，其他
    Calc_UnRestrict(database, datetime1, next_month)  # 解禁
    Calc_UnRestrict_Ratio(database, datetime1, datetime2)  # 解禁比例

    # 新版本函数从20220509 开始计算，未刷新之前数据
    Calc_ShareHoldingChg_Planning_v2(database, datetime1, datetime2)  # 股东增减持

    # ---股票市场：情绪与技术 ---
    Calc_Weekly_Return(database, datetime1, datetime2)
    Calc_BlockTrade(database, datetime1, datetime2)  # 大宗交易
    Calc_Volatility(database, datetime1, datetime2)  # 波动率
    Calc_Turnover(database, datetime1, datetime2)  # 市场情绪-换手率
    Calc_Turnover_Rank(database, datetime1, datetime2)
    Calc_AMT(database, datetime1, datetime2)
    # Calc_Investor_Confidence(database, datetime1, datetime2)  # 20210629 不再更新

    Calc_Industries_Rotation(database, datetime1, datetime2)
    # Calc_Bond_Stock_Rotate(database, datetime1, datetime2) # 部分数据不再更新
    Calc_Bond_Future_Rotate(database, datetime1, datetime2)  # 新公式求，日度数据不求月平均，直接按月求和  # 数据不再更新
    # Calc_Stock_Cap_To_Bond(database, datetime1, datetime2)  # 这个注意05年之前要拼接  # 数据不再更新
    Calc_Bond_Leverage(database, datetime1, datetime2)  # 这个注意05年之前要拼接
    Calc_MoneyMarket_leverage(database, datetime1, datetime2)
    Calc_Low_Price_Stock_Ratio(database, datetime1, datetime2)

    # 股指期货
    # Calc_Basis(database, datetime1, datetime2) # 旧版本，已停止
    # SystematicFactors.Basis.Calc_FutureSpotBasis(database, datetime1, datetime2, "IC") # 旧版本，已停止
    # SystematicFactors.Basis.Calc_FutureSpotBasis(database, datetime1, datetime2, "IF") # 旧版本，已停止
    SystematicFactors.Basis_v2.Calc_Stock_Future_Spot_Basis(database, datetime1, datetime2)

    # 股指期权
    download_option_implied_volatility(database, datetime1, datetime2)
    # SystematicFactors.iVixFactor.Calc_iVix(database, datetime1, datetime2)  # 没有数据

    # 市场结构
    # 市场微观结构
    Calc_UpDown(database, datetime1, datetime2)  # 上涨下跌比例

    # ---周期类因子---
    SystematicFactors.Hurst.Calc_Hurst(database, datetime2, period="Weekly")
    SystematicFactors.Hurst.Calc_Hurst(database, datetime2, period="Monthly")

    # 周期类因子
    # SystematicFactors.DDM_Cycle.Run_DDM(database, datetime1, datetime2)
    SystematicFactors.Kitchin_Cycle.Calc_Kitchin_Cycle(database, datetime2, period="Weekly")
    SystematicFactors.Kitchin_Cycle.Calc_Kitchin_Cycle(database, datetime2, period="Monthly")

    # 待修改函数
    # SystematicFactors.ReturnStructure.Calculate_ReturnStructure(database, realtime, datetime1, datetime2)

    # 低频运行
    # datetime1 = datetime.datetime(2000, 1, 1)
    # datetime2 = datetime.datetime(2020, 5, 11)
    # Calculate_Aggregate_ROE(database, datetime1, datetime2)

    # ZYYX 朝阳永续因子
    # ---Connect zhao yang yong xu Database---
    # zyyx_database = MySQLDB.MySQLDB("172.25.4.214", "3306",
    #                                 username="j_goaldb",
    #                                 password="5JrBqPn9eX8N")
    # Calc_Est_HS300(zyyx_database, database, datetime1, datetime2)
    # Calc_Est_Percentile_HS300(zyyx_database, database, datetime1, datetime2)

    # External-File 因子
    # Load_MF_HoldingLevel(database, pathFilename='C:/Users/fengshimeng3/Documents/财富管理-智能投顾/inputdata/开放式基金估算股票投资比例.xlsx')

    # Basic 因子
    # Complex 因子
    # Advanced 因子


#
def check_systematic_factor(database, excel_path_filename):
    #
    df = pd.read_excel(excel_path_filename, sheet_name="FactorList")



if __name__ == '__main__':
    #
    path_filename = os.getcwd() + "\..\Config\config_local.json"
    database = Config.create_database(database_type="MySQL", config_file=path_filename, config_field="MySQL")
    #
    endDateTime = datetime.datetime.now()
    endDateTime = datetime.datetime(2024, 12, 31)
    #
    # Generate_Systematic_Factor_Dataframe(datetime.datetime(2003,1,1), datetime.datetime(2020,7,1), "CHNBOND")
    # Generate_Systematic_Factor_Dataframe(datetime.datetime(2000, 1, 1), datetime.datetime(2020, 7, 12), "HS300")

    #
    Wind.w.start()

    # 计算系统因子 Systematic Factor
    Automatic_Calculate_Systematic_Factors(database, realtime=None, datetime2=endDateTime, cold_start=False)